# coding=utf-8
import urllib2
from time import sleep
import logging

import proxy

proxy_support = urllib2.ProxyHandler(proxy.getProxy())
opener = urllib2.build_opener(proxy_support)
urllib2.install_opener(opener)
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S',
                    filename='/data/logs/python/python.log',
                    filemode='w')


def download(url, num_retries=3):
    logging.info('Downloading:'+url)
    sleep(3)
    headers = {'User-agent': '+1s'}
    request = urllib2.Request(url, headers=headers)
    try:
        html = urllib2.urlopen(url).read()
    except urllib2.URLError as e:
        logging.error('download error:', e.reason)
        html = None
        if num_retries > 1:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                return download(num_retries - 1)
        else:
            proxy.changeProxy()
            return download(num_retries - 1)
    return html
